# -*- coding: utf-8 -*-
import scrapy
from scrapy import Request
import re
import json
from pprint import pprint


class MoviesSpider(scrapy.Spider):
    BASE_URL = 'https://movie.douban.com/j/search_subjects?type=movie&tag=%s&sort=recommend&page_limit=%s&page_start=%s'
    name = 'movies'
    MOVIE_TAG = '豆瓣高分'
    PAGE_LIMIT = 20
    page_start = 0
    start_urls = [BASE_URL % (MOVIE_TAG, PAGE_LIMIT, page_start)]

    def parse(self, response):
        # 使用json模块解析相应结果
        infos = json.loads(response.body.decode('utf-8'))
        # 迭代影片信息列表
        for movie_info in infos['subjects']:
            movie_item = {}
            movie_item['片名'] = movie_info['title']
            movie_item['评分'] = movie_info['rate']
            yield Request(movie_info['url'], callback=self.parse_movie, meta={'_movie_item': movie_item})
            if len(infos['subjects']) == self.PAGE_LIMIT:
                self.page_start += self.PAGE_LIMIT
                url = self.BASE_URL % (self.MOVIE_TAG, self.PAGE_LIMIT, self.page_start)
                yield Request(url)

    def parse_movie(self,response):
        # 从meta中提取已包含‘片名’和‘评分’的信息item
        movie_item = response.meta['_movie_item']
        # 获取整个信息字符串
        info = response.css('div.subject div#info').xpath('string(.)').extract_first()
        # 提取所有字段名
        fields = [s.strip().replace(':', '') for s in response.css('div#info span.pl::text').extract()]
        # 提取所有字段的值
        values = [re.sub('\s+', '', s.strip()) for s in re.split('\s*(?:%s):\s*' % '|'.join(fields), info)][1:]
        # 将所有信息填入item
        movie_item.update(dict(zip(fields, values)))
        yield movie_item

